from selenium import webdriver
import redis
from hashlib import md5
import time

class MzbSpider:
    def __init__(self):
        self.url = 'http://www.mca.gov.cn/article/sj/xzqh/2020/'
        # redis
        self.r = redis.Redis(host='localhost', port=6380, db=0)
        # 打开浏览器,进入主页
        self.options = webdriver.ChromeOptions()
        self.options.add_argument('--headless')
        self.driver = webdriver.Chrome(options=self.options)
        self.driver.get(url=self.url)

    def md5_s(self, s):
        """md5加密功能函数"""
        m = md5()
        m.update(s.encode())

        return m.hexdigest()

    def parse_html(self):
        """爬虫逻辑函数"""
        new_a = self.driver.find_element_by_partial_link_text("县以上行政区划代码")
        # 增量判断
        new_href = new_a.get_attribute('href')
        finger = self.md5_s(new_href)
        if self.r.sadd('mzb:spiders', finger):
            # 要抓,有更新!
            new_a.click()
            # 给页面加载预留时间
            time.sleep(3)
            # 切换句柄
            li = self.driver.window_handles
            self.driver.switch_to.window(li[1])
            # 提取数据
            tr_list = self.driver.find_elements_by_xpath('//tr[@height="19"]')
            for tr in tr_list:
                L = tr.text.split()
                item = {}
                item['code'] = L[0]
                item['name'] = L[1]
                print(item)

            # 抓取完成后关闭浏览器
            self.driver.quit()
        else:
            print('未更新新月份')
            self.driver.quit()


if __name__ == '__main__':
    spider = MzbSpider()
    spider.parse_html()





























